'''
Created on 2012. 11. 17.

@author: love
'''
import MySQLdb
import threading
import Queue
import time
import gc
import os


data_queue = Queue.Queue()
file_queue = Queue.Queue()

class ThreadFile(threading.Thread):
    def __init__(self, queue, out_queue):
        threading.Thread.__init__(self)
        self.queue = queue
        self.out_queue = out_queue
        os.system('del C:\\workspace\\topics\\contents\\*.* /Q')
        
    def run(self):
        while True:
            (seq, contents) = self.queue.get()                        
            if contents == None:
                raise Exception
            filename = 'C:/workspace/data/contents/%d.txt' % seq
            try:
                f = open(filename, 'w')
                f.write(contents)
                f.close()
                self.out_queue.put(seq)
            except:
                pass
            
            finally:
                self.queue.task_done()
                
class ThreadIndex(threading.Thread):
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue
        os.system('del C:\\workspace\\topics\\index\\*.* /Q')
        
    def run(self):
        while True:
            seq = self.queue.get()
            inputfile = 'C:/workspace/data/contents/%d.txt' % seq
            outputfile = 'C:/workspace/data/index/%d.txt' % seq
            try:
                os.system('index.exe %s %s' % (inputfile, outputfile))                
                
            except Exception as e:
                print e
                
            finally:
                self.queue.task_done()        

def main():       
    db = MySQLdb.connect('127.0.0.1', 'root', 'test', 'trends', charset='utf8', use_unicode=True)
    cursor = db.cursor()
    
    sql = 'select seq, contents from article'
#    sql = 'select seq, contents from article where seq between 512798 and 512838'
    cursor.execute(sql)
    
    for _ in range(5):
        ft = ThreadFile(data_queue, file_queue)
        ft.setDaemon(True)
        ft.start()
        
    for _ in range(5):
        it = ThreadIndex(file_queue)
        it.setDaemon(True)
        it.start()
    
    data = cursor.fetchall()
    for row in data:        
        if row == None:
            continue
        data_queue.put(row)
        
    data_queue.join()
    file_queue.join()
    
    db.close()
    gc.collect()

if __name__ == '__main__':
    start = time.time()
    main()
    print 'Elapsed time : %s' % (time.time() - start)